
                /*********************************************
                **                                          **
                **                                          **
                **                 barrier-r                **
                **                                          **
                **                                          **
                *********************************************/
                /*********************************************
                **                                          **
                **         Author Jean-Marc Adamo           **
                **       Copyright (C) 1997-2008 JMA        **
                *********************************************/


/*
** execute with 6 processes
*/


#include "arch.h"
#include "A.utilities"

//user program

Coll_id cid_bar1;
Coll_id cid1, cid2, cid3;
Coll_id cid_bar2;

//on most homogenous systems, explicit setting 
//wont be necessary (try commenting the
//setting lines, you should get the same values
//for the cids).
void init(int, char **){
   cid_bar1.set();
   cid1.set();
   cid2.set();
   cid3.set();
   cid_bar2.set();
   fprintf(Sdf::trace, "node%d: cid_bar1=%d, cid1=%d, cid2=%d, cid3=%d, cid_bar2=%d\n",
           Sdf::PROC_ID, 
           cid_bar1.get(), 
           cid1.get(), cid2.get(), cid3.get(),
           cid_bar2.get());
   fflush(Sdf::trace);
}

class Proc: public S_Thread{

   public:
        int pid;
        int step;

        Proc(int p, int s)
                : S_Thread((Smf0_ptr)&Proc::body){pid = p; step = s;}

        void body(){
               fprintf(Sdf::trace, "node %d, in proc %d start at step %d\n",
                       Sdf::PROC_ID, pid, step);
               fflush(Sdf::trace);

               int i;
               for(i = 0; i < 4; i++){
                   fprintf(Sdf::trace, "node %d, pid:%d, step:%d, i=%d\n",
                           Sdf::PROC_ID, pid, step, i);
                   fflush(Sdf::trace);

                   reschedule();
               }

               if(step<1){
                   if(pid == 1){
                      fprintf(Sdf::trace,"node%d, ######before barrier, pid=%d\n",
                              Sdf::PROC_ID, pid);
                      fflush(Sdf::trace);

                      Sdf::barrier(cid_bar1);

                      fprintf(Sdf::trace,"node%d, ######after barrier, pid=%d\n",
                              Sdf::PROC_ID, pid);
                      fflush(Sdf::trace);
                   }else{

                      Group *group1;
                      int rk1[3];
                      for(i = 0; i < 3; i++)
                         rk1[i] = i;
                      group1 = Sdf::incl(Sdf::COMML_WORLD, 3, rk1);
                      CommL *comm1 = group1->commit(cid1, Sdf::COMML_WORLD);

                      Group *group2;
                      int rk2[3];
                      for(i = 3; i < Sdf::PROC_NB; i++)
                         rk2[i-3] = i;
                      group2 = Sdf::incl(Sdf::COMML_WORLD, Sdf::PROC_NB-3, rk2);
                      CommL *comm2 = group2->commit(cid2, Sdf::COMML_WORLD);

                      CommR *commr;
                        //Sdf::COMML_WORLD is used as a super group.

                      commr = Sdf::newCommR(cid3, comm1, comm2, Sdf::COMML_WORLD);

                      int *rem_ranks;
                      int *loc_ranks;
                      rem_ranks = commr->get_remote_CommL()->get_world_ranks();
                      loc_ranks = commr->get_local_CommL()->get_world_ranks();

                      int i;
                      int size = commr->get_local_CommL()->size();
                      fprintf(Sdf::trace, "node %d, >>>local  CommL, size=%d\n", 
                              Sdf::PROC_ID, size);
                      for(i=0; i<size; i++)
                         fprintf(Sdf::trace, "node %d,                        loc_ranks[%d]=%d\n", 
                              Sdf::PROC_ID, i, loc_ranks[i]);

                      size = commr->get_remote_CommL()->size();
                      fprintf(Sdf::trace, "node %d, >>>remote CommL, size=%d\n", 
                       Sdf::PROC_ID, size);
                      for(i=0; i<size; i++)
                         fprintf(Sdf::trace, "node %d,                        rem_ranks[%d]=%d\n", 
                                 Sdf::PROC_ID, i, rem_ranks[i]);
                      fflush(Sdf::trace);

                      delete []rem_ranks;
                      delete []loc_ranks;

                      fprintf(Sdf::trace,"node%d, ######before barrier, pid=%d\n",
                              Sdf::PROC_ID, pid);
                      fflush(Sdf::trace);


/* */
                      Sdf::barrier(cid_bar2, commr);
/* */

                      fprintf(Sdf::trace,"node%d, ######after barrier, pid=%d\n",
                              Sdf::PROC_ID, pid);
                      fflush(Sdf::trace);

                      delete comm1;
                      delete comm2;
                      delete commr;

                   }

                   Proc *proc1 = new Proc(pid*10 + 1, step+1);
                   Proc *proc2 = new Proc(pid*10 + 2, step+1);
                   S_Thread *proc_tabl[2] = {proc1, proc2};
                   int pri[2] = {1, 1};
                   par(2, proc_tabl, pri);

                   delete proc1;
                   delete proc2;
               }
               fprintf(Sdf::trace, "on node %d, proc %d the end at step %d\n",
                        Sdf::PROC_ID, pid, step);
               fflush(Sdf::trace);
        }
};

class Root_proc: public Thread{

   public:

        Root_proc()
                : Thread((Tmf0_ptr)&Root_proc::body){}

        void body(){

                fprintf(Sdf::trace, "node %d, in Root_proc::body1\n", Sdf::PROC_ID);
                fflush(Sdf::trace);
                Proc *proc1 = new Proc(1, 0);
                Proc *proc2 = new Proc(2, 0);
                fprintf(Sdf::trace, "node %d, in Root_proc::body2\n", Sdf::PROC_ID);
                fflush(Sdf::trace);
                S_Thread *proc_tabl[2] = {proc1, proc2};
                int pri[2] = {1, 1};
                par(2, proc_tabl, pri);
                fprintf(Sdf::trace, "node %d, in Root_proc::body4\n", Sdf::PROC_ID);
                fflush(Sdf::trace);

                delete proc1;
                delete proc2;
        }
};
//end of user program

MAIN(Root_proc, init, Sdf::empty, Sdf::empty)

